#!/bin/bash

###
### Write By LanXinen Optimize By Stu
### Date:20191105
### Version:0.1
### Description: Install the necessary software, environment, optimize the kernel parameters and install k8s for CentOS7
### History:
###			20191105：
###			1、安装流程化；
###
### 具体安装问题请参考官网
### https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/

## variable define area （变量定义区）
# 指定kubernetes版本
#KUBENETES_VERSION=$(KUBE_VERSION:-1.14.8)
KUBENETES_VERSION="1.14.8"
DOCKER_VERSION="18.09.7"
#MASTER=$(echo $@ | grep "master")
# ！！！ 配置master的时候记得修改v_master_ip，将把IP地址写入hosts ！！！
v_master_ip="172.16.2.132"
v_basic_check_network=""
v_basic_check_domain=""
v_install_log="/tmp/install.log"
#v_first_run=""
#v_basic_run=""
v_id="$(id -u)"
# 当配置参数值为1，将在初始化环境的时候，不进行yum update操作
v_yum_update="0"

v_mark_help=""
v_mark_basic=""
v_mark_master=""
v_mark_node=""
v_mark_dashboard=""
v_mark_ingress=""

v_mark_first_run="/var/run/utaka_auto_install_k8s_first.run"
v_mark_basic_run="/var/run/utaka_auto_install_k8s_basic.run"
v_mark_master_run="/var/run/utaka_auto_install_k8s_master.run"
v_mark_node_run="/var/run/utaka_auto_install_k8s_.run"
v_mark_dashboard_run="/var/run/utaka_auto_install_k8s_dashboard.run"
v_mark_ingress_run="/var/run/utaka_auto_install_k8s_ingress.run"

## func
first_run() {
  echo "Hello, this is your first time to run the automated installation k8s environment script, please check the notes:"
  echo "你好，这是你第一次运行自动化安装k8s环境脚本，请查看一下注意事项："
  echo "1. Please use -h parameter to view the help information of this script."
  echo "1、请使用 -h 参数查看本脚本的帮助信息。"
  echo
  echo "2、Please edit this script, check \" variable definition area \", find the variable :\"v_master_ip\", and change the following IP address to the k8s-master Server IP,if this Server do not need to install k8s-master, please ignore."
  echo "2、请编辑本脚本，查看\"变量定义区\"，找到变量:\"v_master_ip\"，并将后面的IP地址修改为安装master机器的IP地址，本机不需要安装k8s master的话，请忽略。"
  echo "   Current variable value:\"v_master_ip\" value is：$v_master_ip"
  echo "   当前变量值:\"v_master_ip\"值为：$v_master_ip"
  echo
  echo "3、Please edit this script, view \" variable definition area \", find variables: KUBENETES_VERSION, DOCKER_VERSION,and modify the software version you need!"
  echo "3、请编辑本脚本，查看\"变量定义区\"，找到变量：KUBENETES_VERSION，DOCKER_VERSION，修改需要安装的软件版本！"
  echo "   Current kubernetes version：$KUBENETES_VERSION"
  echo "   Current docker version：$DOCKER_VERSION"
  echo "   当前kubernetes版本：$KUBENETES_VERSION"
  echo "   当前docker版本：$DOCKER_VERSION"
  echo
  echo "4、Some lengthy logs are stored in $v_install_log. Note: this file is cleaned before each installation."
  echo "4、部分冗长的日志存放在 $v_install_log ，注意：每次安装前会清空该文件。"
  echo
  echo "5、When the script executes the base environment installation (with-b parameter), it will run \" yum -y update\" to update CentOS to the latest state. If you don't want that, find and modify the value of the variable in \" variable definition area \" to: v_yum_update=1."
  echo "   Current variable value：$v_yum_update"
  echo "5、脚本执行基础环境安装时（使用 -b 参数），将运行yum -y update，将CentOS更新至最新状态，如果不想执行此操作，请在\"变量定义区\"找到并修改对应变量的值为: v_yum_update=1。"
  echo "   当前变量值：$v_yum_update"
  echo "1" >"$v_mark_first_run"
}

basic_check() {
  ping 114.114.114.114 -c 3 >/dev/null 2>&1
  v_basic_check_network="$?"
  ping www.baidu.com -c 3 >/dev/null 2>&1
  v_basic_check_domain="$?"
  if [ "$v_basic_check_network" -ne "0" ]; then
    echo "connecting internet failure , please checking your network connection !"
    exit 111
  elif [ "$v_basic_check_domain" -ne "0" ]; then
    chattr -i /etc/resolv.conf
    chattr -a /etc/resolv.conf
    sed -i '1,$d' /etc/resolv.conf
    echo "nameserver 172.16.2.9" >>/etc/resolv.conf
    chattr +i /etc/resolv.conf
    echo "DNS lookup failure , insert 172.16.2.9 & 218.85.157.99 to resolv.conf"
  elif [[ "$v_id" != "0" ]]; then
    echo "please run this script by root!"
    exit 111
  fi
}

basic_env() {
  # add hosts
  echo "add hosts IP ..."
  echo "$v_master_ip   kube-master" >>/etc/hosts

  # prepare yum repo
  echo "configuring yum repo ..."
  sleep 1
  mkdir -p /etc/yum.repos.d/old >/dev/null 2>&1
  mv /etc/yum.repos.d/*.repo /etc/yum.repos.d/old/ >/dev/null 2>&1
  curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo >/dev/null 2>&1
  curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo >/dev/null 2>&1
  curl -o /etc/yum.repos.d/docker-ce.repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo >/dev/null 2>&1
  cat <<EOF >/etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

  # makecahce for yum
  echo "making up yum repo cache now ..."
  echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" >>$v_install_log 2>&1
  echo "yum makecache fast" >>$v_install_log 2>&1
  echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" >>$v_install_log 2>&1
  yum makecache fast >>$v_install_log 2>&1

  # update system latest
  if [[ "$v_yum_update" != "1" ]]; then
    echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" >>$v_install_log 2>&1
    echo "yum -y update" >>$v_install_log 2>&1
    echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" >>$v_install_log 2>&1
    echo "yum updating now ..."
    yum -y update >>$v_install_log 2>&1
  fi

  # install basic tools ，必须先安装 nfs-utils 才能挂载 nfs 网络存储
  echo "installing basic tools ..."
  sleep 1
  echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" >>$v_install_log 2>&1
  echo "yum install -y yum-utils wget net-tools nfs-utils" >>$v_install_log 2>&1
  echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" >>$v_install_log 2>&1
  yum install -y yum-utils wget net-tools nfs-utils >>$v_install_log 2>&1

  # configure services & function
  echo "configuring services ..."
  sleep 1
  echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" >>$v_install_log 2>&1
  echo "yum remove -y firewalld" >>$v_install_log 2>&1
  echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" >>$v_install_log 2>&1
  systemctl stop firewalld >>$v_install_log 2>&1 && systemctl disable firewalld >>$v_install_log 2>&1 && yum remove -y firewalld >>$v_install_log 2>&1
  setenforce 0
  sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config

  # configure env
  echo "configuring environment ..."
  sleep 1
  swapoff -a
  sed -i '/swap/d' /etc/fstab

  echo "optimizing now ..."
  sleep 1
  sed -i '/\(^#\|^$\)/!d' /etc/security/limits.conf

  # configure ulimit parameter
  cat <<EOF >>/etc/security/limits.conf
root soft nofile 1024000
root hard nofile 1024000
* soft nofile 1024000
* hard nofile 1024000
EOF

  # kernel parameter
  cat <<EOF >>/etc/sysctl.conf
kernel.pid_max = 65535
kernel.threads-max = 200000
vm.max_map_count = 65535
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 4194304
net.core.somaxconn = 65535
net.core.netdev_max_backlog = 262144
net.ipv4.tcp_max_orphans = 262144
net.ipv4.ip_local_port_range=1024 65000
net.ipv4.tcp_tw_recycle=1
net.ipv4.tcp_tw_reuse=1
net.ipv4.tcp_syn_retries=1
net.ipv4.tcp_fin_timeout=30
net.ipv4.tcp_keepalive_time=1200
fs.aio-max-nr=1000000
fs.file-max=1000000
fs.inotify.max_user_watches=1048576
net.ipv4.tcp_keepalive_probes=3
net.ipv4.tcp_keepalive_intvl=15
net.ipv4.tcp_retries1=3
net.ipv4.tcp_retries2=5

net.ipv6.conf.all.disable_ipv6=1
net.ipv6.conf.default.disable_ipv6=1
net.ipv6.conf.lo.disable_ipv6=1

vm.swappiness=0
net.ipv4.neigh.default.gc_stale_time=120

# see details in https://help.aliyun.com/knowledge_detail/39428.html
net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.default.rp_filter=0
net.ipv4.conf.default.arp_announce=2
net.ipv4.conf.lo.arp_announce=2
net.ipv4.conf.all.arp_announce=2

# see details in https://help.aliyun.com/knowledge_detail/41334.html
net.ipv4.tcp_max_tw_buckets=5000
net.ipv4.tcp_syncookies=1
net.ipv4.tcp_max_syn_backlog=1024
net.ipv4.tcp_synack_retries=2
kernel.sysrq=1

EOF

  cat <<EOF >/etc/sysctl.d/99-k8s.conf
kernel.softlockup_all_cpu_backtrace=1
kernel.softlockup_panic=1
vm.max_map_count=262144

net.core.wmem_max=16777216
net.core.somaxconn=32768
net.core.netdev_max_backlog=16384
net.core.rmem_max=16777216

fs.inotify.max_user_watches=524288
fs.file-max=2097152
fs.inotify.max_user_instances=8192
fs.inotify.max_queued_events=16384
fs.may_detach_mounts=1

net.ipv4.tcp_slow_start_after_idle=0
net.ipv4.tcp_wmem=4096 12582912 16777216
net.ipv4.ip_forward=1
net.ipv4.tcp_max_syn_backlog=8096
net.ipv4.tcp_rmem=4096 12582912 16777216
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables = 1
EOF

  sysctl -p >>$v_install_log 2>&1
  echo "Clean up old software ... "
  # cleanup
  echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" >>$v_install_log 2>&1
  echo "yum remove -y docker docker-client docker-client-latest docker-common docker-latest docker-latest-logrotate docker-logrotate docker-selinux docker-engine-selinux docker-engine" >>$v_install_log 2>&1
  echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" >>$v_install_log 2>&1
  yum remove -y docker docker-client docker-client-latest docker-common docker-latest docker-latest-logrotate docker-logrotate docker-selinux docker-engine-selinux docker-engine >>$v_install_log 2>&1
  echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" >>$v_install_log 2>&1
  echo "yum remove -y kubelet kubeadm kubectl" >>$v_install_log 2>&1
  echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" >>$v_install_log 2>&1
  yum remove -y kubelet kubeadm kubectl >>$v_install_log 2>&1
}

install_ipvs_module() {
  echo "Installing ipvs ..."
  # load ipvs
  sleep 1
  cat <<EOF >/etc/sysconfig/modules/ipvs.modules
#!/bin/sh
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
  chmod 755 /etc/sysconfig/modules/ipvs.modules
  modprobe -- ip_vs
  modprobe -- ip_vs_rr
  modprobe -- ip_vs_wrr
  modprobe -- ip_vs_sh
  modprobe -- nf_conntrack_ipv4

  #安装ipvs管理工具
  yum install -y ipset ipvsadm >>$v_install_log
}

install_docker() {
  echo "Installing docker ..."
  #安装指定版本的docker，docker版本通过DOCKER_VERSION变量指定
  echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" >>$v_install_log 2>&1
  echo "yum install -y docker-ce-"$DOCKER_VERSION" docker-ce-cli-"$DOCKER_VERSION" containerd.io" >>$v_install_log 2>&1
  echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" >>$v_install_log 2>&1
  yum install -y docker-ce-"$DOCKER_VERSION" docker-ce-cli-"$DOCKER_VERSION" containerd.io >>$v_install_log 2>&1
  mkdir -p /etc/docker
  cat <<EOF >/etc/docker/daemon.json
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2",
  "storage-opts": [
    "overlay2.override_kernel_check=true"
  ],
  "selinux-enabled": false,
  "registry-mirrors": ["https://k4azpinc.mirror.aliyuncs.com"],
  "bip": "10.16.0.1/16"
}
EOF
  systemctl enable docker >>$v_install_log 2>&1 && systemctl start docker >>$v_install_log 2>&1
}

install_k8s() {
  #安装kubernetes组件
  echo "Installing k8s component ..."
  echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" >>$v_install_log 2>&1
  echo "yum install -y kubelet-${KUBENETES_VERSION}-0 kubeadm-${KUBENETES_VERSION}-0 kubectl-${KUBENETES_VERSION}-0" >>$v_install_log 2>&1
  echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" >>$v_install_log 2>&1
  yum install -y kubelet-${KUBENETES_VERSION}-0 kubeadm-${KUBENETES_VERSION}-0 kubectl-${KUBENETES_VERSION}-0 >>$v_install_log 2>&1
  systemctl enable kubelet >>$v_install_log 2>&1 && systemctl start kubelet >>$v_install_log 2>&1
}

docker_logrotate() {
  echo "Configuring docker logrotate ..."
  sleep 1
  cat <<EOF >/etc/logrotate.d/docker
/var/lib/docker/containers/*/*.log
{
    size    50M
    rotate  0
    missingok
    nocreate
    #compress
    copytruncate
    nodelaycompress
    notifempty
}
EOF
}

get_info() {
  echo "docker version:"
  docker version
  echo "kubectl version:"
  kubectl version
  echo "kubeadm version:"
  kubeadm version
}

get_help() {
  echo "Usage: $0 [OPTION] [ARGUMENT]"
  echo "使用过程发现BUG请反馈"
  echo
  echo
  echo -e -n "\t"
  echo -e -n "\033[31m-b\033[0m"
  echo -e -n "\t"
  echo "该参数进行基础环境配置与安装，不论安装master,node等都需要先配置好基础环境"
  echo -e -n "\t"
  echo -e -n "\033[31m-m\033[0m"
  echo -e -n "\t"
  echo "该参数将安装kubernetes master"
  echo -e -n "\t"
  echo -e -n "\033[31m-n\033[0m"
  echo -e -n "\t"
  echo "该参数将提示安装kubernetes node"
  echo -e -n "\t"
  echo -e -n "\033[31m-i\033[0m"
  echo -e -n "\t"
  echo "该参数将安装kubernetes ingress"
  echo -e -n "\t"
  echo -e -n "\033[31m-d\033[0m"
  echo -e -n "\t"
  echo "该参数将安装kubernetes dashboard"
  echo -e -n "\t"
  echo -e -n "\033[31m-h\033[0m"
  echo -e -n "\t"
  echo "该参数获取此帮助信息"
}

install_k8s_master() {
  echo "installing k8s master ..."
  hostname kube-master >>$v_install_log 2>&1
  hostnamectl set-hostname kube-master >>$v_install_log 2>&1
  echo "$v_master_ip   kube-master" >>/etc/hosts
  # 如下 yaml文件请注意格式，/t 和 空格等，要对齐
  cat <<EOF >./kubeadm.yaml
apiVersion: kubeadm.k8s.io/v1beta1
kind: InitConfiguration
bootstrapTokens:
- groups:
  ttl: "0"
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: "ipvs"
---
apiVersion: kubeadm.k8s.io/v1beta1
imageRepository: "registry.aliyuncs.com/google_containers"
kind: ClusterConfiguration
kubernetesVersion: v${KUBENETES_VERSION}
networking:
  dnsDomain: ${DNS_DOMAIN:-cluster.local}
  podSubnet: ${PAD_SUBNET:-10.244.0.0/16}
  serviceSubnet: ${SERVICE_SUBNET:-10.1.0.0/16}
EOF
  # 初始化
  kubeadm init --config kubeadm.yaml
  # 配置 kubectl
  rm -rf /root/.kube/ >>$v_install_log 2>&1
  mkdir /root/.kube/ >>$v_install_log 2>&1
  cp /etc/kubernetes/admin.conf /root/.kube/config >>$v_install_log 2>&1
  # 配置flannel网络
  curl -o kube-flannel.yml https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml >>$v_install_log 2>&1
  sed -i 's|"Type": "vxlan"|"Type": "vxlan","Directrouting": true|g' kube-flannel.yml
  kubectl apply -f kube-flannel.yml
}

install_dashboard() {
  echo "installing k8s dashboard ..."
  sleep 1
  mkdir -p /root/dashboard-certs
  openssl req -newkey rsa:4096 -nodes -sha256 -keyout /root/dashboard-certs/dashboard.key -x509 -days 365 -out /root/dashboard-certs/dashboard.crt
  cd /root
  kubectl -n kube-system create secret generic kubernetes-dashboard-certs --from-file=/root/dashboard-certs/dashboard.crt --from-file=/root/dashboard-certs/dashboard.key
  # install dashboard
  kubectl apply -f https://raw.githubusercontent.com/inspireso/docker/kubernetes-1.14/kubernetes/google_containers/kubernetes-dashboard1.10.yaml
  # add admin
  kubectl apply -f https://raw.githubusercontent.com/inspireso/docker/kubernetes/kubernetes/google_containers/kubernetes-dashboard-admin.rbac.yaml
  # print token
  kubectl describe secrets -n kube-system $(kubectl -n kube-system get secret | awk '/dashboard-admin/{print $1}')
}

install_ingress() {
  echo "installing ingress now ..."
  sleep 1
  #部署nginx-ingress
  kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.24.1/deploy/mandatory.yaml
  #部署nginx-ingress-service
  kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.24.1/deploy/provider/baremetal/service-nodeport.yaml
}

# prepare stage
echo >$v_install_log
touch $v_mark_first_run
touch $v_mark_basic_run
touch $v_mark_master_run
touch $v_mark_node_run
touch $v_mark_dashboard_run
touch $v_mark_ingress_run
#v_first_run="$(cat $v_mark_first_run)"
#v_basic_run="$(cat $v_mark_basic_run)"

# main

while getopts "mndbih" arg; do
  case $arg in
  h)
    v_mark_help="True"
    ;;
  m)
    v_mark_master="True"
    ;;
  n)
    v_mark_node="True"
    ;;
  d)
    v_mark_dashboard="True"
    ;;
  b)
    v_mark_basic="True"
    ;;
  i)
    v_mark_ingress="True"
    ;;
  ?)
    get_help
    exit 1
    ;;
  esac
done

if [[ "$(cat $v_mark_first_run)" != "1" ]]; then
  first_run
  exit 0
elif [[ "$v_mark_help" == "True" ]] || [[ "$#" == "0" ]]; then
  get_help
elif [[ "$v_mark_basic" == "True" ]]; then
  echo -e "\033[46;30mStage is started , you can view yum log in $v_install_log\033[0m"
  echo
  echo -e "\033[46;30mChecking ...\033[0m"
  basic_check
  echo -e "\033[46;30mEnd of the stage ...\033[0m"
  sleep 1

  echo -e "\033[46;30mSetting Up Basic Environment ...\033[0m"
  basic_env
  echo -e "\033[46;30mEnd of the stage ...\033[0m"
  sleep 1

  echo -e "\033[46;30mloading ipvs module ...\033[0m"
  install_ipvs_module
  echo -e "\033[46;30mEnd of the stage ...\033[0m"
  sleep 1

  echo -e "\033[46;30mInstalling docker ...\033[0m"
  install_docker
  echo -e "\033[46;30mEnd of the stage ...\033[0m"
  sleep 1

  echo -e "\033[46;30mInstalling kubernetes ...\033[0m"
  install_k8s
  echo -e "\033[46;30mEnd of the stage ...\033[0m"
  sleep 1

  echo -e "\033[46;30mLogrotate ...\033[0m"
  docker_logrotate
  echo -e "\033[46;30mEnd of the stage ...\033[0m"
  sleep 1

  echo "1" >$v_mark_basic_run
elif [[ "$(cat $v_mark_basic_run)" != "1" ]]; then
  echo -e "\033[41;30mPlease run the -b parameter before running the other parameters\033[0m"
  exit 1
elif [[ "$v_mark_master" == "True" ]]; then
  echo -e "\033[46;30mInstalling kubernetes ...\033[0m"
  sleep 1
  install_k8s_master
  echo "installtion finished , get cluster info now ..."
  get_info
  echo -e "\033[46;30mEnd of the stage ...\033[0m"
  sleep 1
  echo "1" >$v_mark_master_run
elif [[ "$v_mark_node" == "True" ]]; then
  echo "Make sure you run this options in worker(node)："
  echo "请确认你是在worker服务器上运行-n参数，并按照如下步骤操作，安装部署worker并加入集群："
  echo "1、Running Command In Master : kubeadm token create --config kubeadm.yaml --print-join-command"
  echo "   请在master上执行此命令，记录返回结果，后续步骤需要在worker上执行：kubeadm token create --config kubeadm.yaml --print-join-command"
  echo
  echo "2、Running Command In worker : docker pull registry.cn-hangzhou.aliyuncs.com/ates-k8s/flannel:v0.11.0-amd64"
  echo "   请在worker上执行此命令拉取镜像：docker pull registry.cn-hangzhou.aliyuncs.com/ates-k8s/flannel:v0.11.0-amd64"
  echo
  echo "3、Follow the second running return , like this : kubeadm join --token=xxxxxxxxxxxxx xxx.xxx.xxx.xxx"
  echo "   请在worker上执行第一步返回加入集群的命令，类似：kubeadm join --token=xxxxxxxxxxxxx xxx.xxx.xxx.xxx"
  echo
  echo "   join the worker to the cluster and confirm that the returned result is correct"
  echo "   执行命令后请查看返回内容，确认已成功将worker加入集群"
  echo "1" >$v_mark_node_run
elif [[ "$v_mark_dashboard" == "True" ]]; then
  echo -e "\033[46;30mInstalling dashboard ...\033[0m"
  sleep 1
  install_dashboard
  echo 'installtion is complete ! you can get service-account-token by run this command: '
  echo '安装完成！你可以通过如下命令来获取 service-account-token：'
  echo '   kubectl describe secrets -n kube-system $(kubectl -n kube-system get secret | awk '/dashboard-admin/{print $1}')'
  echo -e "\033[46;30mEnd of the stage ...\033[0m"
  sleep 1
  echo "1" >$v_mark_dashboard_run
elif [[ "$v_mark_ingress" == "True" ]]; then
  echo -e "\033[46;30mInstalling dashboard ...\033[0m"
  sleep 1
  install_ingress
  echo -e "\033[46;30mEnd of the stage ...\033[0m"
  sleep 1
  echo "1" >$v_mark_ingress_run
else
  get_help
fi
